import json
import re
import sys
import time

import facade
from bs4 import BeautifulSoup
from xjlibrary.newthreading.MThreadingRun import MThreadingRun
from xjlibrary.newthreading.mthreading import ThreadManager
from xjlibrary.our_file_dir import BaseDir

curPath = BaseDir.get_file_dir_absolute(__file__)
TopPath = BaseDir.get_upper_dir(curPath, -2)
dirPath = BaseDir.get_new_path(TopPath, "download", "TandBook", "download", "big_json")
configfile = BaseDir.get_new_path(curPath, "db.ini")

# 用于处理键值对的匿名函数
filter_func = lambda key, json_field: json_field[key] if key in json_field and json_field[key] else ''

# 处理字符串的匿名函数
dealtext_func = lambda text: str(text).strip().replace('\n', '').replace("'", "''")

# 处理出版年的匿名函数
dealdate_func = lambda date: str(date).split('-')[0] if date else ''

# 处理出版时间的匿名函数
dealdateC_func = lambda date: str(date).split('T')[0].replace('-', '') if date else ''

# 处理identifier_isbn的匿名函数
dealIdentifier_func = lambda isbn_set: ';'.join(isbn_set) if isbn_set else ''

# 处理不规范的isbn
isbnFilter_func = lambda isbn: isbn if len(str(isbn)) == 13 else 0000000000000

# 处理插入数据库中的引号问题
dealSqlQuote_func = lambda text: text.replace(r"'", r"''") if isinstance(text, str) else ''

# 拼接isbn集合字段
dealIsbnSetFunc = lambda isbn: str(isbn) + '+' if len(str(isbn)) == 13 else '0000000000000+'


class ParseBigJson(object):

    def __init__(self):
        self.logger = facade.get_streamlogger()
        self.mysqlutils = facade.MysqlUtiles(configfile,
                                             "db",
                                             logger=self.logger)

    def set_localVal(self, localVal):
        self.localVal = localVal

    def para(self, threadval, filepath, line, count):
        self.localVal.threadval = threadval
        i = 0
        for each_book_info in self.parseOneLine(line):
            i = i + 1
            # 解析获取到的每本书的isbn集合等信息
            book_eisbn_set, book_pisbn_set, cover_url_set, lngid, book_isbn, title, title_sub, title_edition, creator, publisher, description, provider_subject, page, date, date_created, language, country, provider, provider_url, typeCode, medium, batch = each_book_info
            # 如果isbn为空，则集合肯定为空
            if book_isbn:  # 存在isbn时(部分书籍不存在isbn)
                book_unique_id = self.isExistField(book_pisbn_set, book_eisbn_set)
                # 通过改变行判断是否有更新
                self.updateOrInsertField(book_unique_id, book_eisbn_set, book_pisbn_set, cover_url_set, lngid,
                                         book_isbn,
                                         title, title_sub, title_edition, creator,
                                         publisher, description, provider_subject, page, date, date_created, language,
                                         country, provider, provider_url, typeCode, medium, batch)
            log_str = 'Current Subject: %s, Item Success line: %s,num: %s,id: %s' % (
                filepath.split('.')[0], count, i, lngid)
            self.logger.info(log_str)

    # 更新或插入解析得到的新数据
    def updateOrInsertField(self, book_unique_id, book_eisbn_set, book_pisbn_set, cover_url_set, lngid, book_isbn,
                            title,
                            title_sub, title_edition, creator, publisher, description,
                            provider_subject, page, date, date_created, language, country, provider, provider_url,
                            typeCode,
                            medium, batch):
        # print(batch)
        provider_id = provider + "@" + book_isbn
        book_eisbn_set1 = list(book_eisbn_set)
        book_pisbn_set1 = list(book_pisbn_set)
        book_eisbn_set1.sort()
        book_pisbn_set1.sort()
        identifier_eisbn = dealIdentifier_func(book_eisbn_set1)
        identifier_pisbn = dealIdentifier_func(book_pisbn_set1)
        if book_unique_id:  # 已存在该字段，进行更新操作
            Sql = "UPDATE allfields SET title='{title}', title_sub='{title_sub}', title_edition='{title_edition}', identifier_pisbn='{identifier_pisbn}', identifier_eisbn='{identifier_eisbn}', creator='{creator}', publisher='{publisher}', description='{description}', provider_subject='{provider_subject}', page='{page}', `date`='{date}',date_created='{date_created}', `language`='{language}', country='{country}', provider='{provider}', provider_url='{provider_url}', `type`='{type}', medium='{medium}' WHERE rawid='{rawid}'"
            Sql = Sql.format(rawid=book_unique_id, title=title, title_sub=title_sub, title_edition=title_edition,
                             identifier_pisbn=identifier_pisbn, identifier_eisbn=identifier_eisbn,
                             creator=creator, publisher=publisher, description=description,
                             provider_subject=provider_subject, page=page, date=date, date_created=date_created,
                             language=language, country=country, provider=provider,
                             provider_url=provider_url, type=typeCode, medium=medium)
            # resultbool, rows = self.mysqlutils.ExeSqlToDB(Sql)
            self.localVal.threadval.result_queue.put(Sql)
            # # 通过返回的rows判断是否有更新
            # if resultbool:
            #     if rows > 0:
            #         sql = "update allfields set batch='{batch}'WHERE rawid='{rawid}'"
            #         sql = sql.format(rawid=book_unique_id, batch=batch)
            #         # resultbool, rows = self.mysqlutils.ExeSqlToDB(sql)
            #         self.localVal.threadval.result_queue.put(sql)
            #         assert resultbool, ValueError("更新失败,请检查，并手动将该句插入数据库 保证数据不会漏")

        else:  # 不存在该字段则插入新的数据
            book_isbn_set = book_eisbn_set | book_pisbn_set
            if book_isbn_set is not None:
                identifier_eisbn = dealIdentifier_func(book_eisbn_set1)
                identifier_pisbn = dealIdentifier_func(book_pisbn_set1)

                Sql_filed = "INSERT IGNORE INTO allfields(lngid, rawid, title, title_sub, title_edition, identifier_pisbn, identifier_eisbn, creator, publisher, `description`, provider_subject, `page`, `date`,date_created, `language`, country, provider, provider_url,provider_id, type, `medium`, batch)"
                Sql_values = " VALUES('%s', '%s','%s', '%s','%s','%s', '%s','%s', '%s','%s','%s','%s', '%s','%s', '%s','%s', '%s','%s', '%s','%s','%s','%s')" % (
                    lngid, book_isbn, title, title_sub, title_edition, identifier_pisbn, identifier_eisbn, creator,
                    publisher, description, provider_subject, page, date,
                    date_created, language, country, provider, provider_url, provider_id, typeCode, medium, batch)
                Sql = Sql_filed + Sql_values
                # resultbool, rows = self.mysqlutils.ExeSqlToDB(Sql)
                self.localVal.threadval.result_queue.put(Sql)
        # 更新MySQL数据库中的封面url
        self.saveCoverUrl2DB(book_isbn, cover_url_set)

    # 保存isbn码已经对应的封面url字符串至MySQL数据库
    def saveCoverUrl2DB(self, book_isbn, cover_url_set):
        cover_url_set = list(cover_url_set)
        cover_url_set.sort()
        cover_url_str = '**'.join(cover_url_set)
        # Select_Sql = "SELECT isbn13,ava_url FROM cover WHERE isbn13='%s'" % book_isbn
        # item_result = self.mysqlutils.SelectFromDB(Select_Sql)
        if book_isbn in self.cover_dicts.keys():
            if not self.cover_dicts[book_isbn] == cover_url_str:
                Update_Sql = "UPDATE cover SET url='%s' WHERE isbn13='%s'" % (cover_url_str, book_isbn)
                self.localVal.threadval.result_queue.put(Update_Sql)
                self.logger.info('Update cover.url:%s' % book_isbn)
        else:
            Insert_Sql = "INSERT IGNORE INTO cover(isbn13, url) VALUES('%s','%s')" % (book_isbn, cover_url_str)
            self.localVal.threadval.result_queue.put(Insert_Sql)
            self.logger.info('Insert cover.url:%s' % book_isbn)

    # 判断数据库中是否已存在该条数据
    def isExistField(self, pisbn_set, eisbn_set):
        """
        :param mysql_conn: db3数据库连接对象
        :param pisbn_set: pisbn集合
        :param eisbn_set: eisbn集合
        :return: rawid 或 None
        """
        # 并集
        all_isbn_set = pisbn_set | eisbn_set
        isbn_values_str = '|'.join(map(dealIsbnSetFunc, all_isbn_set))
        # Sql = "SELECT * FROM modify_title_info_zt WHERE identifier_pisbn REGEXP '{isbn_set}' OR identifier_eisbn REGEXP '{isbn_set}'"
        self.logger.info("isbn_values_str is : {}".format(isbn_values_str))
        # Sql = "SELECT rawid FROM allfields WHERE identifier_pisbn REGEXP '{isbn_set}' OR identifier_eisbn REGEXP '{isbn_set}'"
        # Sql = Sql.format(isbn_set=isbn_values_str)
        # self.logger.debug(Sql)
        # result = self.mysqlutils.SelectFromDB(Sql)

        regex = r'(#[^&]+({})&)'.format(isbn_values_str)
        result = re.findall(regex, self.isbn_all)
        if result:
            rawid = result[0][0].split(',')[0].replace('#', '')
            self.logger.info("rawid is : {}".format(rawid))
            return rawid
        return None

        # # 如果存在该数据字段，返回该字段的rawid
        # if result and result[0][0]:
        #     print(result[0][0])
        #     return result[0][0]
        # # 若不存在该字段，说明是新数据，则返回None
        # return None

    # 获取当天日期
    def getTodayDate(self):
        return time.strftime('%Y%m%d', time.localtime(time.time()))

    # 设置公司规定的一些信息
    def setCompanyInfo(self, isbn):
        lngid = 'TANDF_TS_' + isbn
        provider = 'tandfbook'
        provider_url = provider + '@https://www.taylorfrancis.com/books/' + isbn
        typeCode = 1  # 图书
        medium = 2  # 数字
        batch = self.getTodayDate() + '00'  # 设置批次
        # print(lngid, provider, provider_url, type, medium, batch)
        return lngid, provider, provider_url, typeCode, medium, batch

    # 解析单行的数据
    def parseOneLine(self, line):
        line_json = json.loads(line, encoding='utf-8')
        # 这里返回了10条数据
        for each_book in line_json['data']['resultSet']:
            # book_isbn 是一个 不是一个set
            book_isbn, cover_url_set, book_eisbn_set, book_pisbn_set = self.getISBNandCover(each_book['formats'])
            # 这里的isbn很多
            book_isbn_set = book_eisbn_set | book_pisbn_set  # 书籍总集合为ebook+纸质书并集
            if book_isbn_set is not None:
                title, title_sub, title_edition, page, date, date_created, publisher, provider_subject, creator, description, language, country = self.bookOtherInfo(
                    each_book)
                lngid, provider, provider_url, typeCode, medium, batch = self.setCompanyInfo(book_isbn)
                yield book_eisbn_set, book_pisbn_set, cover_url_set, lngid, book_isbn, title, title_sub, title_edition, creator, publisher, description, provider_subject, page, date, date_created, language, country, provider, provider_url, typeCode, medium, batch

    # 获取类别关键词
    def getCategoriesToSubjects(self, categories):
        subject_list = []
        if categories:
            for each_sub in categories:
                level = filter_func('level', each_sub)
                if level == 1 and filter_func('text', each_sub):
                    subject_list.append(filter_func('text', each_sub))
        subject = ';'.join(subject_list)
        return subject

    # 获取作者名
    def getAutherName(self, author_info):
        if isinstance(author_info, list):
            author_list = []
            originators = filter_func('originators', author_info[0])
            if originators and isinstance(originators, list):
                for each_author in originators:
                    inner_name = filter_func('name', each_author)
                    if inner_name is not None:
                        author_name = filter_func('full', inner_name)  # 可能不存在full这个key
                        if author_name:
                            author_list.append(author_name)
                        else:
                            author_name = filter_func('first', inner_name) + ' ' + filter_func('last',
                                                                                               inner_name)  # 不存在full这个key时使用名字拼接
                            author_list.append(author_name)
                author_set = set(author_list)
                return ';'.join(author_set)
        return ''

    # 获取摘要
    def getDescription(self, description_info):
        if isinstance(description_info, str) and description_info:
            soup = BeautifulSoup(description_info, 'html.parser')
            description = soup.get_text()
            return description
        else:
            return ''

    # 获取书籍的其他信息
    def bookOtherInfo(self, each_book):
        title = filter_func('title', each_book)  # 标题
        title_sub = filter_func('subtitle', each_book)  # 副标题
        title_edition = filter_func('edition', each_book)  # 出版版本
        page = filter_func('pages', each_book)  # 页码
        date_info = filter_func('firstPublishedOn', each_book)  # 出版年
        if not date_info:
            date_info = filter_func('datePublication', each_book)  # 次要出版年
        if not date_info:
            date_info = '1900-00-00T00:00:00.000Z'  # 当两者出版年份都不存在时取默认值
        date = dealdate_func(date_info)
        date_created = dealdateC_func(date_info)
        publisher = filter_func('imprint', each_book)  # 出版社
        categories_info = filter_func('categories', each_book)
        # subject = getCategoriesToSubjects(categories=categories_info)  # 关键词
        provider_subject = self.getCategoriesToSubjects(categories=categories_info)  # 所属类别
        author_info = filter_func('originators', each_book)
        creator = self.getAutherName(author_info=author_info)  # 作者
        description_info = filter_func('description', each_book)
        description = self.getDescription(description_info)  # 摘要

        language = 'EN'
        country = 'UK'
        return dealtext_func(title), dealtext_func(title_sub), dealtext_func(title_edition), dealtext_func(
            page), dealtext_func(date), dealtext_func(date_created), dealtext_func(
            publisher), dealtext_func(provider_subject), dealtext_func(creator), dealtext_func(
            description), dealtext_func(
            language), dealtext_func(country)

    # 获取isbn和封面img列表(传入为单本书所有类型)
    def getISBNandCover(self, formats_json):
        eisbn_list = []
        pisbn_list = []
        img_cover_list = []
        if formats_json is not None:
            for each in formats_json:
                # 先获取封面img(有很多图片路径)
                if 'coverImages' in each.keys() and each['coverImages']:
                    # extend() 函数用于在列表末尾一次性追加另一个序列中的多个值（用新列表扩展原来的列表）。
                    img_cover_list.extend(each['coverImages'])
                # 首先取eBook中的isbn
                if 'versionTypeCode' in each.keys() and each['versionTypeCode'] == 'EBK':  # 首选EBOOK中的isbn字段
                    if 'isbnPdf' in each and each['isbnPdf']:  # 有eBook的情况下首选isbnPdf
                        eisbn_list.append(each['isbnPdf'].replace('-', ''))
                    if 'isbn13' in each and each['isbn13']:  # 无isbnPdf的情况下选isbn13
                        eisbn_list.append(each['isbn13'].replace('-', ''))
                    if 'isbnMobi' in each and each['isbnMobi']:  # eBook中最后选isbnMobi
                        eisbn_list.append(each['isbnMobi'].replace('-', ''))
                # 选取其他eBook中的isbn
                if 'versionTypeCode' in each.keys() and each['versionTypeCode'] == 'AEBK':  # 首选EBOOK中的isbn字段
                    if 'isbnPdf' in each and each['isbnPdf']:  # 有eBook的情况下首选isbnPdf
                        eisbn_list.append(each['isbnPdf'].replace('-', ''))
                    if 'isbn13' in each and each['isbn13']:  # 无isbnPdf的情况下选isbn13
                        eisbn_list.append(each['isbn13'].replace('-', ''))
                    if 'isbnMobi' in each and each['isbnMobi']:  # eBook中最后选isbnMobi
                        eisbn_list.append(each['isbnMobi'].replace('-', ''))
                # 获取纸质书籍精装本中的isbn
                if 'versionTypeCode' in each.keys() and (
                        each['versionTypeCode'] == 'HARD' or each['versionTypeCode'] == 'PAPE'):
                    if 'isbnPdf' in each and each['isbnPdf']:  # 当无eBook的情况下选择其它字段中的isbnPdf
                        pisbn_list.append(each['isbnPdf'].replace('-', ''))
                    if 'isbn13' in each and each['isbn13']:
                        pisbn_list.append(each['isbn13'].replace('-', ''))
                    if 'isbnMobi' in each and each['isbnMobi']:
                        pisbn_list.append(each['isbnMobi'].replace('-', ''))
            isbn_list = eisbn_list + pisbn_list  # eBook_isbn在前
            if isbn_list:
                return isbn_list[0], set(img_cover_list), set(eisbn_list), set(pisbn_list)

            else:
                return '', set(), set(), set()
        else:
            return '', set(), set(), set()


class ParseThreadRun(MThreadingRun):

    def __init__(self, num):
        super(ParseThreadRun, self).__init__(num)
        self.parse = ParseBigJson()
        self.count = 0
        self.parse.set_localVal(self.thread_pool.localVal)
        self.select_all()
        self.batch = self.parse.getTodayDate() + '00'

    def select_all(self):
        sql = "select rawid,identifier_pisbn,identifier_eisbn from allfields"
        rows = self.parse.mysqlutils.SelectFromDB(sql)
        lists = []
        for rawid, identifier_pisbn, identifier_eisbn in rows:
            lists.append("".join(["#", rawid, ",", identifier_pisbn, ";", identifier_eisbn, "&"]))
        isbn_all = "".join(lists)
        # self.isbn_all = isbn_all
        self.parse.isbn_all = isbn_all
        sql = "SELECT isbn13,ava_url FROM cover"
        cover_rows = self.parse.mysqlutils.SelectFromDB(sql)
        cover_dicts = {}
        for isbn13, url in cover_rows:
            cover_dicts[isbn13] = url
        # self.cover_dicts = cover_dicts
        self.parse.cover_dicts = cover_dicts

    def getTask(self, *args, **kwargs):
        pass

    def setTask(self, results=None, *args, **kwargs):
        for filepath in BaseDir.get_dir_all_files(dirPath):
            file = open(filepath, encoding='utf-8')
            while True:
                self.count += 1
                line = file.readline()
                self.add_job(self.func, filepath, line, self.count)
                if not line:
                    break
        return "break"

    def setProxy(self, proxysList=None):
        return "break"

    def is_break(self):
        return True

    def thread_pool_hook(self, thread_pool_dicts: dict, dictsfull: dict, *args, **kwargs):
        if dictsfull["threadname"] in ["task_thread_1", "proxy_thread_1"]:
            thread = dictsfull["thread"]
            if not isinstance(thread, ThreadManager):
                raise TypeError("线程对象不是ThreadManager")
            thread.is_restart = False

    def fun(self, threadval, *args, **kwargs):
        self.parse.para(threadval, args[0], args[1], args[2])

    def dealresult(self, *args, **kwargs):
        for sql in self.results:
            if sql.startswith("UPDATE allfields SET"):
                rawid = sql.split("rawid=")[1].replace("'", "")
                resultbool, rows = self.parse.mysqlutils.ExeSqlToDB(sql)
                if resultbool:
                    if rows > 0:
                        sql = "update allfields set batch='{batch}'WHERE rawid='{rawid}'"
                        sql = sql.format(rawid=rawid, batch=self.batch)
                        self.parse.mysqlutils.ExeSqlToDB(sql)
            else:
                self.parse.mysqlutils.ExeSqlToDB(sql)


def main():
    threadrun = ParseThreadRun(20)
    threadrun.run(model=2)


if __name__ == "__main__":
    main()
